屈折|词性_NLTK朴素贝叶斯,文本分类代码示例采样SMSSpamCollection数据集下载

篇首语：本文由编程笔记#小编为大家整理，主要介绍了NLTK朴素贝叶斯,文本分类代码示例 采样 SMSSpamCollection数据集下载相关的知识，希望对你有一定的参考价值。

https://stackoverflow.com/

文本分类代码 编译通过

import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import csv
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
import sklearn.metrics as metrics
from sklearn import tree
from sklearn.linear_model import SGDClassifier
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
# 文本清洗预处理
def preprocessing(text):
# text &＃61; text.encode("gbk").decode("utf8")
# tokons to word 句子标记解析 单词标记解析
tokens &＃61; [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
# print("单词标记解析&＃xff1a;", tokens)
# 停用词删除
stop &＃61; stopwords.words(&＃39;english&＃39;)
tokens &＃61; [token for token in tokens if token not in stop]
# print("停用词删除", tokens)
# 单词字数小于3删除 并转换成小写
tokens &＃61; [word.lower() for word in tokens if len(word) >&＃61; 3]
# print("字数小于3删除", tokens)
# lemmatize
# 把&＃xff08;文中的词&＃xff09;按屈折变化形式&＃xff08;或异体形式&＃xff09;进行归类
lemter &＃61; WordNetLemmatizer()
tokens &＃61; [lemter.lemmatize(word) for word in tokens]
# print("词性归类", tokens)
preprosses_text &＃61; " ".join(tokens)
# print("处理后的文本&＃xff1a;", preprosses_text)
return preprosses_text
with open(&＃39;SMSSpamCollection&＃39;, &＃39;r&＃39;, newline&＃61;&＃39;&＃39;, encoding&＃61;&＃39;mac_roman&＃39;) as csvfile:
smsdata_data &＃61; []
sms_lablel &＃61; []
csv_reader &＃61; csv.reader(csvfile, delimiter&＃61;&＃39;\\t&＃39;)
for line in csv_reader:
sms_lablel.append(line[0])
smsdata_data.append(preprocessing(line[1]))
csvfile.close()
print(&＃39;smsdata_data&＃39;, smsdata_data)
print(&＃39;sms_lablel&＃39;, sms_lablel)
# 采样
# 训练集和样本集分开 7&＃xff1a;3
trainset_size &＃61; int(round(len(smsdata_data)*0.70))
# i chose this threshold for 70:30 train and test split.
print(&＃39;The training set size for this classifier is &＃39; &＃43; str(trainset_size) &＃43; &＃39;\\n&＃39;)
x_train &＃61; np.array([&＃39;&＃39;.join(el) for el in smsdata_data[0:trainset_size]])
y_train &＃61; np.array([el for el in sms_lablel[0:trainset_size]])
x_test &＃61; np.array([&＃39;&＃39;.join(el) for el in smsdata_data[trainset_size&＃43;1:len(smsdata_data)]])
# or el in sms_labels[trainset_size&＃43;1:len(sms_lablel)]])
y_test &＃61; np.array([el for el in sms_lablel[trainset_size&＃43;1:len(sms_lablel)]])
print("x_train&＃xff1a;&＃61;&＃61;&＃61;&＃61;", x_train)
print("y_train:&＃61;&＃61;&＃61;&＃61;", y_train)
# 术语文档矩阵 bow 词袋
with open(&＃39;SMSSpamCollection&＃39;, &＃39;r&＃39;, newline&＃61;&＃39;&＃39;, encoding&＃61;&＃39;mac_roman&＃39;) as csvfile:
sms_exp &＃61; []
csv_reader &＃61; csv.reader(csvfile, delimiter&＃61;&＃39;\\t&＃39;)
for line in csv_reader:
sms_exp.append(preprocessing(line[1]))
vectorizer &＃61; CountVectorizer(min_df&＃61;1)
X_exp &＃61; vectorizer.fit_transform(sms_exp)
print("||".join(vectorizer.get_feature_names()))
print(&＃39;X_exp>>>>>&＃39;, X_exp.toarray())
csvfile.close()
# TF/IDF
vectorizer &＃61; TfidfVectorizer(min_df&＃61;2, ngram_range&＃61;(1, 2), stop_words&＃61;&＃39;english&＃39;, strip_accents&＃61;&＃39;unicode&＃39;, norm&＃61;&＃39;l2&＃39;)
X_train &＃61; vectorizer.fit_transform(x_train)
X_test &＃61; vectorizer.transform(x_test)
print(&＃39;x_train:.....>>>>&＃39;, X_train)
print(&＃39;x_test:>>>>>>>&＃39;, X_test)
# 朴素贝叶斯 朴素贝叶斯分类器
clf &＃61; MultinomialNB().fit(X_train, y_train)
y_nb_predicted &＃61; clf.predict(X_test)
print("y_nb_predicted>>", y_nb_predicted)
print(&＃39;\\n confusion_matrix \\n &＃39;)
cm &＃61; metrics.confusion_matrix(y_test, y_nb_predicted)
print(cm)
print(&＃39;\\n Here is the classification report:&＃39;)
print(metrics.classification_report(y_test, y_nb_predicted))
# 得到前n个特征值
feature_names &＃61; vectorizer.get_feature_names()
coefs &＃61; clf.coef_
intercept &＃61; clf.intercept_
coefs_with_fns &＃61; sorted(zip(clf.coef_[0], feature_names))
n &＃61; 15
top &＃61; zip(coefs_with_fns[:n], coefs_with_fns[:-(n &＃43; 1):-1])
for (coef_1, fn_1), (coef_2, fn_2) in top:
print(&＃39;\\t%.4f\\t%-15s\\t\\t%.4f\\t%-15s&＃39; % (coef_1, fn_1, coef_2, fn_2))
# 决策树 分类器
# clf &＃61; tree.DecisionTreeClassifier.fit(X_train.toarray(), y_train)
# y_tree_predicted &＃61; clf.predict(X_test.toarray())
# print(y_tree_predicted)
# print(&＃39; \\n Here is the classification report: y_tree_predicted&＃39;)
# print(metrics.classification_report(y_test, y_tree_predicted))
# 随机梯度下降
clf &＃61; SGDClassifier(alpha&＃61;0.001, max_iter&＃61;50).fit(X_train, y_train)
y_pred &＃61; clf.predict(X_test)
print(&＃39;\\n Here is the classification report:&＃39;)
print(metrics.classification_report(y_test, y_pred))
print(&＃39; \\n confusion_matrix \\n &＃39;)
cm &＃61; (metrics.confusion_matrix(y_test, y_pred))
print(cm)
# 支持向量机
svm_classifier &＃61; LinearSVC().fit(X_train, y_train)
y_svm_predicted &＃61; svm_classifier.predict(X_test)
print(&＃39;\\n Here is the classification report:&＃39;)
print(metrics.classification_report(y_test, y_svm_predicted))
print(&＃39; \\n confusion_matrix \\n &＃39;)
cm &＃61; (metrics.confusion_matrix(y_test, y_svm_predicted))
print(cm)
# 随机森林
# RandomForestClassifier
clf &＃61; RandomForestClassifier(n_estimators&＃61;10)
clf.fit(X_train, y_train)
y_RF_pred &＃61; clf.predict(X_test)
print(&＃39;RF_confusion_matrix:&＃39;)
print(metrics.confusion_matrix(y_test, y_RF_pred))
print(&＃39;RF_classification_report:&＃39;)
print(metrics.classification_report(y_test, y_RF_pred))

UCI垃圾邮件数据集下载

http://archive.ics.uci.edu/ml/datasets/SMS&＃43;Spam&＃43;Collection